Skip to contentSkip to conversation

utils.py: add parallel make helpers

PATCH In review Joshua Watt 3 participants 6 messages Feb 12, 2018 – Feb 18, 2018

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 utils.py: add parallel make helpers

The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.

The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.

Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.

---
 meta/classes/go.bbclass              | 19 +---------------
 meta/classes/waf.bbclass             | 24 +-------------------
 meta/lib/oe/utils.py                 | 43 ++++++++++++++++++++++++++++++++++++
 meta/recipes-core/ovmf/ovmf_git.bb   |  2 +-
 meta/recipes-support/boost/boost.inc | 24 +-------------------
 5 files changed, 47 insertions(+), 65 deletions(-)
meta/classes/go.bbclass +1 -18
index 09b01a84c37..7ecd8c92541 100644
@@ -1,23 +1,6 @@
 inherit goarch ptest
 
-def get_go_parallel_make(d):
-    pm = (d.getVar('PARALLEL_MAKE') or '').split()
-    # look for '-j' and throw other options (e.g. '-l') away
-    # because they might have a different meaning in golang
-    while pm:
-        opt = pm.pop(0)
-        if opt == '-j':
-            v = pm.pop(0)
-        elif opt.startswith('-j'):
-            v = opt[2:].strip()
-        else:
-            continue
-
-        return '-p %d' % int(v)
-
-    return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
 
 GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
 GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
meta/classes/waf.bbclass +1 -23
index bdbdc56767c..f9a851d0759 100644
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""
 
 EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
 
-def get_waf_parallel_make(d):
-    pm = d.getVar('PARALLEL_MAKE')
-    if pm:
-        # look for '-j' and throw other options (e.g. '-l') away
-        # because they might have different meaning in bjam
-        pm = pm.split()
-        while pm:
-            v = None
-            opt = pm.pop(0)
-            if opt == '-j':
-                v = pm.pop(0)
-            elif opt.startswith('-j'):
-                v = opt[2:].strip()
-            else:
-                v = None
-
-            if v:
-                v = min(64, int(v))
-                return '-j' + str(v)
-
-    return ""
-
 python waf_preconfigure() {
     import subprocess
     from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {
 
 do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
 waf_do_compile()  {
-	${S}/waf build ${@get_waf_parallel_make(d)}
+	${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
 }
 
 waf_do_install() {
meta/lib/oe/utils.py +43 -0

49 lines not on this page · raw message

meta/recipes-core/ovmf/ovmf_git.bb +1 -1
index fa0d66291d1..8750b3c528d 100644
@@ -151,7 +151,7 @@ do_compile_class-native() {
 
 do_compile_class-target() {
     export LFLAGS="${LDFLAGS}"
-    PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+    PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
     OVMF_ARCH="X64"
     if [ "${TARGET_ARCH}" != "x86_64" ] ; then
         OVMF_ARCH="IA32"
meta/recipes-support/boost/boost.inc +1 -23
index 41fc90fb211..0461ec6fcf1 100644
@@ -135,29 +135,7 @@ BJAM_TOOLS   = "--ignore-site-config \
 
 # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j
 # https://svn.boost.org/trac/boost/ticket/7634
-def get_boost_parallel_make(d):
-    pm = d.getVar('PARALLEL_MAKE')
-    if pm:
-        # look for '-j' and throw other options (e.g. '-l') away
-        # because they might have different meaning in bjam
-        pm = pm.split()
-        while pm:
-            v = None
-            opt = pm.pop(0)
-            if opt == '-j':
-                v = pm.pop(0)
-            elif opt.startswith('-j'):
-                v = opt[2:].strip()
-            else:
-                v = None
-
-            if v:
-                v = min(64, int(v))
-                return '-j' + str(v)
-
-    return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
 BJAM_OPTS    = '${BOOST_PARALLEL_MAKE} -d+2 -q \
 		${BJAM_TOOLS} \
 		-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
Signature · 1 line
2.14.3
#1 lore raw

+ v = max(limit, v)

this seems to be doing the opposite of what the limit was used for before:

- v = min(64, int(v))

Quoted 219 lines · Joshua Watt

The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.

The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.

Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.

---
 meta/classes/go.bbclass              | 19 +---------------
 meta/classes/waf.bbclass             | 24 +-------------------
 meta/lib/oe/utils.py                 | 43 ++++++++++++++++++++++++++++++
++++++
 meta/recipes-core/ovmf/ovmf_git.bb   |  2 +-
 meta/recipes-support/boost/boost.inc | 24 +-------------------
 5 files changed, 47 insertions(+), 65 deletions(-)
meta/classes/go.bbclass +1 -18
index 09b01a84c37..7ecd8c92541 100644
@@ -1,23 +1,6 @@
 inherit goarch ptest
 
-def get_go_parallel_make(d):
-    pm = (d.getVar('PARALLEL_MAKE') or '').split()
-    # look for '-j' and throw other options (e.g. '-l') away
-    # because they might have a different meaning in golang
-    while pm:
-        opt = pm.pop(0)
-        if opt == '-j':
-            v = pm.pop(0)
-        elif opt.startswith('-j'):
-            v = opt[2:].strip()
-        else:
-            continue
-
-        return '-p %d' % int(v)
-
-    return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
 
 GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
 GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
meta/classes/waf.bbclass +1 -23
index bdbdc56767c..f9a851d0759 100644
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""
 
 EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
 
-def get_waf_parallel_make(d):
-    pm = d.getVar('PARALLEL_MAKE')
-    if pm:
-        # look for '-j' and throw other options (e.g. '-l') away
-        # because they might have different meaning in bjam
-        pm = pm.split()
-        while pm:
-            v = None
-            opt = pm.pop(0)
-            if opt == '-j':
-                v = pm.pop(0)
-            elif opt.startswith('-j'):
-                v = opt[2:].strip()
-            else:
-                v = None
-
-            if v:
-                v = min(64, int(v))
-                return '-j' + str(v)
-
-    return ""
-
 python waf_preconfigure() {
     import subprocess
     from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {
 
 do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
 waf_do_compile()  {
-       ${S}/waf build ${@get_waf_parallel_make(d)}
+       ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
 limit=64)}
 }
 

waf_do_install() {

meta/lib/oe/utils.py +0 -0
index 7a79d752b69..ec91927233a 100644
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1",

falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue,
falsevalue, d)

meta/lib/oe/utils.py +3 -0
+def parallel_make(d):
+    """
+    Return the integer value for the number of parallel threads to use

when
+ building, scraped out of PARALLEL_MAKE. If no parallelization option
is

meta/lib/oe/utils.py +22 -0
+    found, returns None
+
+    e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
+    """
+    pm = (d.getVar('PARALLEL_MAKE') or '').split()
+    # look for '-j' and throw other options (e.g. '-l') away
+    while pm:
+        opt = pm.pop(0)
+        if opt == '-j':
+            v = pm.pop(0)
+        elif opt.startswith('-j'):
+            v = opt[2:].strip()
+        else:
+            continue
+
+        return int(v)
+
+    return None
+
+def parallel_make_argument(d, fmt, limit=None):
+    """
+    Helper utility to construct a parallel make argument from the number

of

meta/lib/oe/utils.py +3 -0
+    parallel threads specified in PARALLEL_MAKE.
+
+    Returns the input format string `fmt` where a single '%d' will be

expanded
+ with the number of parallel threads to use. If limit is specified,
the
+ number of parallel threads will be no larger than it. If no
parallelization

meta/lib/oe/utils.py +3 -0
+    option is found in PARALLEL_MAKE, returns an empty string
+
+    e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d")

will return

meta/lib/oe/utils.py +9 -0
+    "-n 10"
+    """
+    v = parallel_make(d)
+    if v:
+        if limit:
+            v = max(limit, v)
+        return fmt % v
+    return ''
+
 def packages_filter_out_system(d):
     """
     Return a list of packages from PACKAGES with the "system" packages

such as

meta/recipes-core/ovmf/ renamed from meta/recipes-core/ovmf/ovmf_git.bb +0 -0

ovmf_git.bb
index fa0d66291d1..8750b3c528d 100644

meta/recipes-core/ovmf/ovmf_git.bb +1 -1
@@ -151,7 +151,7 @@ do_compile_class-native() {
 
 do_compile_class-target() {
     export LFLAGS="${LDFLAGS}"
-    PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+    PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
     OVMF_ARCH="X64"
     if [ "${TARGET_ARCH}" != "x86_64" ] ; then
         OVMF_ARCH="IA32"
meta/recipes-support/boost/boost.inc +0 -0
index 41fc90fb211..0461ec6fcf1 100644
@@ -135,29 +135,7 @@ BJAM_TOOLS   = "--ignore-site-config \
 
 # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater

parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634

meta/recipes-support/boost/boost.inc +1 -23
-def get_boost_parallel_make(d):
-    pm = d.getVar('PARALLEL_MAKE')
-    if pm:
-        # look for '-j' and throw other options (e.g. '-l') away
-        # because they might have different meaning in bjam
-        pm = pm.split()
-        while pm:
-            v = None
-            opt = pm.pop(0)
-            if opt == '-j':
-                v = pm.pop(0)
-            elif opt.startswith('-j'):
-                v = opt[2:].strip()
-            else:
-                v = None
-
-            if v:
-                v = min(64, int(v))
-                return '-j' + str(v)
-
-    return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d',

limit=64)}"

 BJAM_OPTS    = '${BOOST_PARALLEL_MAKE} -d+2 -q \
                ${BJAM_TOOLS} \
                -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \

--
2.14.3

--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core

#2 lore raw
Quoted 9 lines · Martin Jansa

+ v = max(limit, v)

this seems to be doing the opposite of what the limit was used for before:

- v = min(64, int(v))

Hmm, you're right

On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected]> wrote:
The code to extract the integer number of parallel build threads and

construct a new argument from them has started to be copied in multiple

locations, so create two new helper utilities to aid recipes.

The first helper (parallel_make()) extracts the integer number of

parallel build threads from PARALLEL_MAKE. The second

(parallel_make_argument()) does the same and then puts the result back

into a format string, optionally clamping it to some maximum value.

Additionally, rework the oe-core recipes that were manually doing this

to use the new helper utilities.

---

 meta/classes/go.bbclass              | 19 +---------------
 meta/classes/waf.bbclass             | 24 +-------------------
 meta/lib/oe/utils.py                 | 43 ++++++++++++++++++++++++++++++++++++
 meta/recipes-core/ovmf/ovmf_git.bb   |  2 +-
 meta/recipes-support/boost/boost.inc | 24 +-------------------
 5 files changed, 47 insertions(+), 65 deletions(-)
meta/classes/go.bbclass +0 -0

index 09b01a84c37..7ecd8c92541 100644

--- a/meta/classes/go.bbclass

+++ b/meta/classes/go.bbclass

@@ -1,23 +1,6 @@

inherit goarch ptest

-def get_go_parallel_make(d):

- pm = (d.getVar('PARALLEL_MAKE') or '').split()

- # look for '-j' and throw other options (e.g. '-l') away

- # because they might have a different meaning in golang

- while pm:

- opt = pm.pop(0)

- if opt == '-j':

- v = pm.pop(0)

- elif opt.startswith('-j'):

- v = opt[2:].strip()

- else:

- continue

-

- return '-p %d' % int(v)

-

- return ""

-

-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"

+GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"

GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"

GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"

meta/classes/waf.bbclass +0 -0

index bdbdc56767c..f9a851d0759 100644

--- a/meta/classes/waf.bbclass

+++ b/meta/classes/waf.bbclass

@@ -3,28 +3,6 @@ DISABLE_STATIC = ""

EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"

-def get_waf_parallel_make(d):

- pm = d.getVar('PARALLEL_MAKE')

- if pm:

- # look for '-j' and throw other options (e.g. '-l') away

- # because they might have different meaning in bjam

- pm = pm.split()

- while pm:

- v = None

- opt = pm.pop(0)

- if opt == '-j':

- v = pm.pop(0)

- elif opt.startswith('-j'):

- v = opt[2:].strip()

- else:

- v = None

-

- if v:

- v = min(64, int(v))

- return '-j' + str(v)

-

- return ""

-

python waf_preconfigure() {

import subprocess

from distutils.version import StrictVersion

@@ -47,7 +25,7 @@ waf_do_configure() {

do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"

waf_do_compile() {

- ${S}/waf build ${@get_waf_parallel_make(d)}

+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}

}

waf_do_install() {

meta/lib/oe/utils.py +0 -0

index 7a79d752b69..ec91927233a 100644

--- a/meta/lib/oe/utils.py

+++ b/meta/lib/oe/utils.py

@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):

"""

return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)

+def parallel_make(d):

+ """

+ Return the integer value for the number of parallel threads to use when

+ building, scraped out of PARALLEL_MAKE. If no parallelization option is

+ found, returns None

+

+ e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.

+ """

+ pm = (d.getVar('PARALLEL_MAKE') or '').split()

+ # look for '-j' and throw other options (e.g. '-l') away

+ while pm:

+ opt = pm.pop(0)

+ if opt == '-j':

+ v = pm.pop(0)

+ elif opt.startswith('-j'):

+ v = opt[2:].strip()

+ else:

+ continue

+

+ return int(v)

+

+ return None

+

+def parallel_make_argument(d, fmt, limit=None):

+ """

+ Helper utility to construct a parallel make argument from the number of

+ parallel threads specified in PARALLEL_MAKE.

+

+ Returns the input format string fmt where a single '%d' will be expanded

+ with the number of parallel threads to use. If limit is specified, the

+ number of parallel threads will be no larger than it. If no parallelization

+ option is found in PARALLEL_MAKE, returns an empty string

+

+ e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return

+ "-n 10"

+ """

+ v = parallel_make(d)

+ if v:

+ if limit:

+ v = max(limit, v)

+ return fmt % v

+ return ''

+

def packages_filter_out_system(d):

"""

Return a list of packages from PACKAGES with the "system" packages such as

meta/recipes-core/ovmf/ovmf_git.bb +0 -0

index fa0d66291d1..8750b3c528d 100644

--- a/meta/recipes-core/ovmf/ovmf_git.bb

+++ b/meta/recipes-core/ovmf/ovmf_git.bb

@@ -151,7 +151,7 @@ do_compile_class-native() {

do_compile_class-target() {

     export LFLAGS="${LDFLAGS}"

- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"

+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"

     OVMF_ARCH="X64"

     if [ "${TARGET_ARCH}" != "x86_64" ] ; then

         OVMF_ARCH="IA32"
meta/recipes-support/boost/boost.inc +0 -0

index 41fc90fb211..0461ec6fcf1 100644

--- a/meta/recipes-support/boost/boost.inc

+++ b/meta/recipes-support/boost/boost.inc

@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \

 # use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j

# https://svn.boost.org/trac/boost/ticket/7634

-def get_boost_parallel_make(d):

- pm = d.getVar('PARALLEL_MAKE')

- if pm:

- # look for '-j' and throw other options (e.g. '-l') away

- # because they might have different meaning in bjam

- pm = pm.split()

- while pm:

- v = None

- opt = pm.pop(0)

- if opt == '-j':

- v = pm.pop(0)

- elif opt.startswith('-j'):

- v = opt[2:].strip()

- else:

- v = None

-

- if v:

- v = min(64, int(v))

- return '-j' + str(v)

-

- return ""

-

-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"

+BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"

BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \

                ${BJAM_TOOLS} \

                -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \

--

2.14.3

--

_______________________________________________

Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core

#3 lore raw
In reply to Martin Jansa
Quoted 6 lines · Martin Jansa

+            v = max(limit, v)

this seems to be doing the opposite of what the limit was used for before:

-                v = min(64, int(v))

moreover, I think the limit should be based on ncpu() instead of hard coding it.

Quoted 227 lines · On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected]…

The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.

The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.

Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.

<mailto:[email protected]>>
---
 meta/classes/go.bbclass              | 19 +--------------- meta/classes/waf.bbclass             | 24 +------------------- meta/lib/oe/utils.py                 | 43
++++++++++++++++++++++++++++++++++++
 meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>   |  2 +- meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)

diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index 09b01a84c37..7ecd8c92541 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
 inherit goarch ptest

-def get_go_parallel_make(d):
-    pm = (d.getVar('PARALLEL_MAKE') or '').split()
-    # look for '-j' and throw other options (e.g. '-l') away -    # because they might have a different meaning in golang -    while pm:
-        opt = pm.pop(0)
-        if opt == '-j':
-            v = pm.pop(0)
-        elif opt.startswith('-j'):
-            v = opt[2:].strip()
-        else:
-            continue
-
-        return '-p %d' % int(v)
-
-    return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"

 GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass index bdbdc56767c..f9a851d0759 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""

 EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"

-def get_waf_parallel_make(d):
-    pm = d.getVar('PARALLEL_MAKE')
-    if pm:
-        # look for '-j' and throw other options (e.g. '-l') away -        # because they might have different meaning in bjam -        pm = pm.split()
-        while pm:
-            v = None
-            opt = pm.pop(0)
-            if opt == '-j':
-                v = pm.pop(0)
-            elif opt.startswith('-j'):
-                v = opt[2:].strip()
-            else:
-                v = None
-
-            if v:
-                v = min(64, int(v))
-                return '-j' + str(v)
-
-    return ""
-
 python waf_preconfigure() {
     import subprocess
     from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {

 do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" waf_do_compile()  {
-       ${S}/waf build ${@get_waf_parallel_make(d)}
+       ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
 }

 waf_do_install() {
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 7a79d752b69..ec91927233a 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
     """
     return bb.utils.contains_any("DISTRO_FEATURES", features,
truevalue, falsevalue, d)

+def parallel_make(d):
+    """
+    Return the integer value for the number of parallel threads to use when
+    building, scraped out of PARALLEL_MAKE. If no parallelization option is
+    found, returns None
+
+    e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer. +    """
+    pm = (d.getVar('PARALLEL_MAKE') or '').split()
+    # look for '-j' and throw other options (e.g. '-l') away +    while pm:
+        opt = pm.pop(0)
+        if opt == '-j':
+            v = pm.pop(0)
+        elif opt.startswith('-j'):
+            v = opt[2:].strip()
+        else:
+            continue
+
+        return int(v)
+
+    return None
+
+def parallel_make_argument(d, fmt, limit=None):
+    """
+    Helper utility to construct a parallel make argument from the number of
+    parallel threads specified in PARALLEL_MAKE.
+
+    Returns the input format string fmt where a single '%d' will be expanded
+    with the number of parallel threads to use. If limit is specified, the
+    number of parallel threads will be no larger than it. If no parallelization
+    option is found in PARALLEL_MAKE, returns an empty string +
+    e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
+    "-n 10"
+    """
+    v = parallel_make(d)
+    if v:
+        if limit:
+            v = max(limit, v)
+        return fmt % v
+    return ''
+
 def packages_filter_out_system(d):
     """
     Return a list of packages from PACKAGES with the "system"
packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
index fa0d66291d1..8750b3c528d 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
+++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
@@ -151,7 +151,7 @@ do_compile_class-native() {

 do_compile_class-target() {
     export LFLAGS="${LDFLAGS}"
-    PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" +    PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
     OVMF_ARCH="X64"
     if [ "${TARGET_ARCH}" != "x86_64" ] ; then OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc
index 41fc90fb211..0461ec6fcf1 100644
--- a/meta/recipes-support/boost/boost.inc
+++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS   = "--ignore-site-config \

 # use PARALLEL_MAKE to speed up the build, but limit it by -j 64,
greater parallelism causes bjam to segfault or to ignore -j
 # https://svn.boost.org/trac/boost/ticket/7634
<https://svn.boost.org/trac/boost/ticket/7634>
-def get_boost_parallel_make(d):
-    pm = d.getVar('PARALLEL_MAKE')
-    if pm:
-        # look for '-j' and throw other options (e.g. '-l') away -        # because they might have different meaning in bjam -        pm = pm.split()
-        while pm:
-            v = None
-            opt = pm.pop(0)
-            if opt == '-j':
-                v = pm.pop(0)
-            elif opt.startswith('-j'):
-                v = opt[2:].strip()
-            else:
-                v = None
-
-            if v:
-                v = min(64, int(v))
-                return '-j' + str(v)
-
-    return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
 BJAM_OPTS    = '${BOOST_PARALLEL_MAKE} -d+2 -q \
                ${BJAM_TOOLS} \
                -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \
--
2.14.3

--
_______________________________________________
Openembedded-core mailing list [email protected] <mailto:[email protected]>
http://lists.openembedded.org/mailman/listinfo/openembedded-core
<http://lists.openembedded.org/mailman/listinfo/openembedded-core>

#4 lore raw

On Feb 17, 2018 18:25, "Khem Raj" <[email protected]> wrote:

Quoted 7 lines · Martin Jansa

+ v = max(limit, v)

this seems to be doing the opposite of what the limit was used for before:

- v = min(64, int(v))

moreover, I think the limit should be based on ncpu() instead of hard coding it.

I don't know about that. Doesn't PARALLEL_MAKE already default to something based on ncpu()? My understanding of the limit was that some recipes (boost? waf?) have a limit on the amount of parallism they can reasonably stand, and you don't want those recipes to break if someone sets PARALLEL_MAKE="-j
1000".

As long as that is true I think this is correct. Whether those recipes still need to limit their parallism is another discussion.

On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected] <mailto:

Quoted 233 lines

[email protected]>> wrote:

The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes.

The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.

Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.

<mailto:[email protected]>>

---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py | 43
++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-

meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)

diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index 09b01a84c37..7ecd8c92541 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest

-def get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE') or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p
%d')}"

GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass index bdbdc56767c..f9a851d0759 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""

EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"

-def get_waf_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
python waf_preconfigure() {
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {

do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" waf_do_compile() {
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
limit=64)}
}

waf_do_install() {
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 7a79d752b69..ec91927233a 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features,
truevalue, falsevalue, d)

+def parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to
use when
+ building, scraped out of PARALLEL_MAKE. If no parallelization
option is
+ found, returns None
+
+ e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an
integer.

    +    """
    +    pm = (d.getVar('PARALLEL_MAKE') or '').split()
    +    # look for '-j' and throw other options (e.g. '-l') away
    +    while pm:

+ opt = pm.pop(0)
+ if opt == '-j':
+ v = pm.pop(0)
+ elif opt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+def parallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the
number of
+ parallel threads specified in PARALLEL_MAKE.
+
+ Returns the input format string fmt where a single '%d' will
be expanded
+ with the number of parallel threads to use. If limit is
specified, the
+ number of parallel threads will be no larger than it. If no
parallelization
+ option is found in PARALLEL_MAKE, returns an empty string
+
+ e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n
%d") will return
+ "-n 10"
+ """
+ v = parallel_make(d)
+ if v:
+ if limit:
+ v = max(limit, v)
+ return fmt % v
+ return ''
+
def packages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system"
packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
index fa0d66291d1..8750b3c528d 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
+++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>

@@ -151,7 +151,7 @@ do_compile_class-native() {

do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc index 41fc90fb211..0461ec6fcf1 100644
--- a/meta/recipes-support/boost/boost.inc +++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \

# use PARALLEL_MAKE to speed up the build, but limit it by -j 64,
greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
<https://svn.boost.org/trac/boost/ticket/7634>
-def get_boost_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam

\
    --
    2.14.3

    --
    _______________________________________________
    Openembedded-core mailing list
    [email protected]
    <mailto:[email protected]>
    http://lists.openembedded.org/mailman/listinfo/openembedded-core
    <http://lists.openembedded.org/mailman/listinfo/openembedded-core>
Quoted 1 line

--

_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core

#5 lore raw
Quoted 19 lines · Joshua Watt

On Feb 17, 2018 18:25, "Khem Raj" <[email protected]> wrote:

Quoted 7 lines · Martin Jansa

+ v = max(limit, v)

this seems to be doing the opposite of what the limit was used for before:

- v = min(64, int(v))

moreover, I think the limit should be based on ncpu() instead of hard coding it.

Quoted 6 lines

I don't know about that. Doesn't PARALLEL_MAKE already default to
something based on ncpu()? My understanding of the limit was that some
recipes (boost? waf?) have a limit on the amount of parallism they can reasonably stand, and you don't want those recipes to break if someone sets PARALLEL_MAKE="-j 1000".

Ah yes that’s a fair point

Quoted 249 lines

As long as that is true I think this is correct. Whether those recipes still need to limit their parallism is another discussion.

On Mon, Feb 12, 2018 at 8:39 PM, Joshua Watt <[email protected]

Quoted 234 lines

<mailto:[email protected]>> wrote:

The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in
multiple
locations, so create two new helper utilities to aid recipes.

The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value.

Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities.

<mailto:[email protected]>>

---
meta/classes/go.bbclass | 19 +---------------
meta/classes/waf.bbclass | 24 +-------------------
meta/lib/oe/utils.py | 43
++++++++++++++++++++++++++++++++++++
meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb> | 2 +-

meta/recipes-support/boost/boost.inc | 24 +------------------- 5 files changed, 47 insertions(+), 65 deletions(-)

diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass index 09b01a84c37..7ecd8c92541 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
inherit goarch ptest

-def get_go_parallel_make(d):
- pm = (d.getVar('PARALLEL_MAKE') or '').split()
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have a different meaning in golang
- while pm:
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- continue
-
- return '-p %d' % int(v)
-
- return ""
-
-GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}" +GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p
%d')}"

GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass index bdbdc56767c..f9a851d0759 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""

EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"

-def get_waf_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
python waf_preconfigure() {
import subprocess
from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {

do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" waf_do_compile() {
- ${S}/waf build ${@get_waf_parallel_make(d)}
+ ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d',
limit=64)}
}

waf_do_install() {
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py index 7a79d752b69..ec91927233a 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
"""
return bb.utils.contains_any("DISTRO_FEATURES", features,
truevalue, falsevalue, d)

+def parallel_make(d):
+ """
+ Return the integer value for the number of parallel threads to
use when
+ building, scraped out of PARALLEL_MAKE. If no parallelization
option is
+ found, returns None
+
+ e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an
integer.

    +    """
    +    pm = (d.getVar('PARALLEL_MAKE') or '').split()
    +    # look for '-j' and throw other options (e.g. '-l') away
    +    while pm:

+ opt = pm.pop(0)
+ if opt == '-j':
+ v = pm.pop(0)
+ elif opt.startswith('-j'):
+ v = opt[2:].strip()
+ else:
+ continue
+
+ return int(v)
+
+ return None
+
+def parallel_make_argument(d, fmt, limit=None):
+ """
+ Helper utility to construct a parallel make argument from the
number of
+ parallel threads specified in PARALLEL_MAKE.
+
+ Returns the input format string fmt where a single '%d' will
be expanded
+ with the number of parallel threads to use. If limit is
specified, the
+ number of parallel threads will be no larger than it. If no
parallelization
+ option is found in PARALLEL_MAKE, returns an empty string
+
+ e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n
%d") will return
+ "-n 10"
+ """
+ v = parallel_make(d)
+ if v:
+ if limit:
+ v = max(limit, v)
+ return fmt % v
+ return ''
+
def packages_filter_out_system(d):
"""
Return a list of packages from PACKAGES with the "system"
packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
index fa0d66291d1..8750b3c528d 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>
+++ b/meta/recipes-core/ovmf/ovmf_git.bb <http://ovmf_git.bb>

@@ -151,7 +151,7 @@ do_compile_class-native() {

do_compile_class-target() {
export LFLAGS="${LDFLAGS}"
- PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}"
+ PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
OVMF_ARCH="X64"
if [ "${TARGET_ARCH}" != "x86_64" ] ; then
OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc index 41fc90fb211..0461ec6fcf1 100644
--- a/meta/recipes-support/boost/boost.inc +++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \

# use PARALLEL_MAKE to speed up the build, but limit it by -j 64,
greater parallelism causes bjam to segfault or to ignore -j
# https://svn.boost.org/trac/boost/ticket/7634
<https://svn.boost.org/trac/boost/ticket/7634>
-def get_boost_parallel_make(d):
- pm = d.getVar('PARALLEL_MAKE')
- if pm:
- # look for '-j' and throw other options (e.g. '-l') away
- # because they might have different meaning in bjam
- pm = pm.split()
- while pm:
- v = None
- opt = pm.pop(0)
- if opt == '-j':
- v = pm.pop(0)
- elif opt.startswith('-j'):
- v = opt[2:].strip()
- else:
- v = None
-
- if v:
- v = min(64, int(v))
- return '-j' + str(v)
-
- return ""
-
-BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}" +BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
${BJAM_TOOLS} \
-sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam

\
    --
    2.14.3

    --
    _______________________________________________
    Openembedded-core mailing list
    [email protected]
    <mailto:[email protected]>
    http://lists.openembedded.org/mailman/listinfo/openembedded-core
    <http://lists.openembedded.org/mailman/listinfo/openembedded-core>
Quoted 1 line

--

_______________________________________________
Openembedded-core mailing list
[email protected]

http://lists.openembedded.org/mailman/listinfo/openembedded-core